package exp;

import java.io.File;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;

import tools.ObjectTools;

/**
 * Divide the urls in batches to avoid using too much memory for crawling
 * 
 * @author lzhou
 * 
 */
public class EXP3_3 {

	public void run(String inputPath, String outputPath, int batch) {
		for (File file : (new File(inputPath)).listFiles()) {
			@SuppressWarnings("unchecked")
			LinkedHashMap<String, String> urls = (LinkedHashMap<String, String>) ObjectTools.readFromFile(file);
			ArrayList<String> w = new ArrayList<>();
			w.addAll(urls.keySet());
			
			int start = 0;
			int end = start + batch;
			while (end < w.size()) {
				List<String> oneBatch = new ArrayList<>(2000);
				System.out.println("Splitting " + file.getName()
						+ " from " + start + " to " + (end - 1));
				for (int i = start; i < end; i++) {
					oneBatch.add(w.get(i));
				}
				ObjectTools.writeToFile(oneBatch, outputPath + file.getName() + "_"
						+ start + "_" + (end - 1));

				start = end;
				end = start + batch;
			}
			if (end > w.size()) {
				List<String> oneBatch = new ArrayList<>();
				end = w.size() - 1;
				System.out.println("Splitting " + file.getName()
						+ " from " + start + " to " + end);
				for (int i = start; i < end; i++) {
					oneBatch.add(w.get(i));
				}
				ObjectTools.writeToFile(oneBatch, outputPath + file.getName() + "_"
						+ start + "_" + end);
			}
		}
	}
	
	public static void main(String[] args) {
		String inputPath = "data/_newData/obj/urlContent/";
		String outputPath2 = "data/_newData/obj/urlContent1/";
		int batch = 2000;
		
		EXP3_3 exp3 = new EXP3_3();
		exp3.run(inputPath, outputPath2, batch);
	}
}
